//
//  NSString+Extras.m
//  SEOBox
//
//  Created by Syd on 10-3-2.
//  Copyright 2010 MusicPeace.ORG. All rights reserved.
//

#import "NSString+Extras.h"
#import "NSArray+Extras.h"
#import "TFHpple.h"
#import <scws.h>
#import <RegexKit/RegexKit.h>

@implementation NSString (Extras)

- (BOOL)containsString:(NSString *)aString {
    return [self containsString:aString ignoringCase:NO];
}

- (BOOL)containsString:(NSString *)aString ignoringCase:(BOOL)flag {
    unsigned mask = (flag ? NSCaseInsensitiveSearch : 0);
    return [self rangeOfString:aString options:mask].length > 0;
}

- (int)countSubstring:(NSString *)aString ignoringCase:(BOOL)flag {
    unsigned mask = (flag ? NSCaseInsensitiveSearch : 0);
    return [self rangeOfString:aString options:mask].length;
}

- (NSString *)stringByTrimmingWhitespace
{
    return [self stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]];
}

#pragma mark HTML STRIP
- (NSString *)flattenHTMLAndTrim:(BOOL)trim {
    
    NSString *title = [self htmlTitle];
    NSString *metaDesc = [self htmlMetaDescription];
    NSArray  *metaKeywords = [self htmlMetaKeywords];
    self = [self removeJavascriptAndTrim:trim];
    self = [self removeCSSAndTrim:trim];
    
    NSScanner *theScanner;
    NSString *text = nil;
    
    theScanner = [NSScanner scannerWithString:self];
    
    while ([theScanner isAtEnd] == NO) {
        
        // find start of tag
        [theScanner scanUpToString:@"<" intoString:NULL] ;                 
        // find end of tag         
        [theScanner scanUpToString:@">" intoString:&text] ;
        
        // replace the found tag with a space
        //(you can filter multi-spaces out later if you wish)
        self = [self stringByReplacingOccurrencesOfString:
                [ NSString stringWithFormat:@"%@>", text]
                                               withString:@""];
        
    } // while //
    
    // trim off whitespace
    self =  trim ? [self stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] : self;
    return [NSString stringWithFormat:@"%@\n%@\n%@\n%@", title, metaDesc, [metaKeywords componentsJoinedByString:@","], self];
}

- (NSString *)removeJavascriptAndTrim:(BOOL)trim {
    
    NSScanner *theScanner;
    NSString *text = nil;
    
    theScanner = [NSScanner scannerWithString:self];
    
    while ([theScanner isAtEnd] == NO) {
        
        // find start of tag
        [theScanner scanUpToString:@"<script" intoString:NULL] ;                 
        // find end of tag         
        [theScanner scanUpToString:@"</script>" intoString:&text] ;
        
        // replace the found tag with a space
        //(you can filter multi-spaces out later if you wish)
        self = [self stringByReplacingOccurrencesOfString:
                [ NSString stringWithFormat:@"%@", text]
                                               withString:@""];
        
    } // while //
    
    // trim off whitespace
    return trim ? [self stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] : self;
    
}

- (NSString *)removeCSSAndTrim:(BOOL)trim {
    
    NSScanner *theScanner;
    NSString *text = nil;
    
    theScanner = [NSScanner scannerWithString:self];
    
    while ([theScanner isAtEnd] == NO) {
        
        // find start of tag
        [theScanner scanUpToString:@"<style" intoString:NULL] ;                 
        // find end of tag         
        [theScanner scanUpToString:@"</style>" intoString:&text] ;
        
        // replace the found tag with a space
        //(you can filter multi-spaces out later if you wish)
        self = [self stringByReplacingOccurrencesOfString:
                [ NSString stringWithFormat:@"%@", text]
                                               withString:@""];
        
    } // while //
    
    // trim off whitespace
    return trim ? [self stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceAndNewlineCharacterSet]] : self;
    
}

#pragma mark TOKENS
- (NSMutableArray *)stringTokens {
    NSMutableArray *results = [[NSMutableArray alloc] init];
    NSString *dictUTF8 = [[NSBundle mainBundle] pathForResource:@"dict.utf8" ofType:@"xdb"];
    NSString *dictChtUTF8 = [[NSBundle mainBundle] pathForResource:@"dict_cht.utf8" ofType:@"xdb"];
    NSString *rulesUTF8 = [[NSBundle mainBundle] pathForResource:@"rules_cht.utf8" ofType:@"ini"];
    //NSString *rulesChtUTF8 = [[NSBundle mainBundle] pathForResource:@"rules_cht.utf8" ofType:@"ini"];
    scws_t s = scws_new();
    scws_set_charset(s, "utf8");
    scws_set_dict(s, [dictUTF8 UTF8String], SCWS_XDICT_XDB);
    scws_add_dict(s, [dictChtUTF8 UTF8String], SCWS_XDICT_XDB);
    scws_set_rule(s, [rulesUTF8 UTF8String]);
    //scws_set_debug(s, SCWS_YEA);
    scws_send_text(s, [self UTF8String], strlen([self UTF8String]));
    int tlimit=0;
    char *attr = NULL;
    scws_top_t top, xtop;
    if ((top = xtop = scws_get_tops(s, tlimit, attr)) != NULL)
    {
        tlimit = 1;
        while (xtop != NULL)
        {
            //printf("%02d. %-24.24s %-4.2s  %.2f(%d)\n", tlimit, xtop->word, xtop->attr, xtop->weight, xtop->times);
            NSString *token = [NSString stringWithUTF8String:xtop->word];
            NSArray *keys = [[NSArray alloc] initWithObjects:@"name", @"weight", @"times", nil];
            NSArray *objs = [[NSArray alloc] initWithObjects:token, [NSNumber numberWithFloat:xtop->weight], [NSNumber numberWithInt:xtop->times], nil];
            NSDictionary *dict = [[NSDictionary alloc] initWithObjects:objs forKeys:keys];
            [results addObject:dict];
            [keys release];
            [objs release];
            [dict release];
            
            xtop = xtop->next;
            tlimit++;
        }
        scws_free_tops(top);
    }
    scws_free(s);
    return results;
}

- (float) IDFForToken:(NSString *)aToken
{
    NSMutableArray *tokens = [[self stringTokens] autorelease];
    int total = 0;
    NSDictionary *t;
    for(t in tokens)
    {
        total += [[t objectForKey:@"times"] intValue];
    }
    int count = [self countSubstring:aToken ignoringCase:TRUE];
    if (count == 0) {
        return 0.0f;
    }
    return log10(total/count);
}

- (float) TFForToken:(NSString *)aToken
{
    int count = [self countSubstring:aToken ignoringCase:TRUE];
    return [aToken length] * count / [self length];
}

- (float) weightForToken: (NSString *)aToken
{
    return [self TFForToken:aToken]*[self IDFForToken:aToken];
}

#pragma mark HTML ELEMENTS
- (NSString *)htmlTitle {
    NSString *title = NULL;
    id regx = [RKRegex regexWithRegexString:@"<title>([^^]*?)</title>" options:RKCompileCaseless];
    [self getCapturesWithRegexAndReferences:regx, @"$1", &title, nil];
    return title;
}

- (NSString *)htmlMetaDescription {
    NSString *desc = NULL;
    id regx = [RKRegex regexWithRegexString:@"<meta\\s+name=[\"|']description[\"|']\\s+content=[\"|']([^^]*?)[\"|']" options:RKCompileCaseless];
    [self getCapturesWithRegexAndReferences:regx, @"$1", &desc, nil];
    return desc;
}

- (NSArray *)htmlMetaKeywords {
    NSString *keywords = NULL;
    id regx = [RKRegex regexWithRegexString:@"<meta\\s+name=[\"|']keywords[\"|']\\s+content=[\"|']([^^]*?)[\"|']" options:RKCompileCaseless];
    [self getCapturesWithRegexAndReferences:regx, @"$1", &keywords, nil];
    return [keywords componentsSeparatedByString:@","];
}
@end
